{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载路径\n",
    "import os\n",
    "SPAM_PATH = os.path.join('datasets','spam') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "HAM_DIR = os.path.join(SPAM_PATH, 'easy_ham')  # 分别获取 正常邮件 与 垃圾邮件 路径，拼接路径\n",
    "SPAM_DIR = os.path.join(SPAM_PATH, 'spam')\n",
    "ham_filenames = [name for name in sorted(os.listdir(HAM_DIR)) if len(name)>20]   \n",
    "spam_filenames = [name for name in sorted(os.listdir(SPAM_DIR)) if len(name)>20]\n",
    "\n",
    "# 列表生产式 获取文件名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 我们可以使用python的“email”模块解析这些电子邮件（它处理邮件头、编码等）\n",
    "\n",
    "import email\n",
    "import email.policy\n",
    "\n",
    "def load_email(is_spam, filename, spam_path = SPAM_PATH):\n",
    "    directory = 'spam' if is_spam else 'easy_ham'    # 正常邮件还是垃圾邮件\n",
    "    with open(os.path.join(spam_path, directory, filename), 'rb') as f:  # 路径拼接，‘rb’ 读取二进制\n",
    "        return email.parser.BytesParser(policy=email.policy.default).parse(f)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Klez: The Virus That Won't Die\n",
      " \n",
      "Already the most prolific virus ever, Klez continues to wreak havoc.\n",
      "\n",
      "Andrew Brandt\n",
      ">>From the September 2002 issue of PC World magazine\n",
      "Posted Thursday, August 01, 2002\n",
      "\n",
      "\n",
      "The Klez worm is approaching its seventh month of wriggling across \n",
      "the Web, making it one of the most persistent viruses ever. And \n",
      "experts warn that it may be a harbinger of new viruses that use a \n",
      "combination of pernicious approaches to go from PC to PC.\n",
      "\n",
      "Antivirus software makers Symantec and McAfee both report more than \n",
      "2000 new infections daily, with no sign of letup at press time. The \n",
      "British security firm MessageLabs estimates that 1 in every 300 \n",
      "e-mail messages holds a variation of the Klez virus, and says that \n",
      "Klez has already surpassed last summer's SirCam as the most prolific \n",
      "virus ever.\n",
      "\n",
      "And some newer Klez variants aren't merely nuisances--they can carry \n",
      "other viruses in them that corrupt your data.\n",
      "\n",
      "...\n",
      "\n",
      "http://www.pcworld.com/news/article/0,aid,103259,00.asp\n",
      "_______________________________________________\n",
      "Irregulars mailing list\n",
      "Irregulars@tb.tf\n",
      "http://tb.tf/mailman/listinfo/irregulars\n"
     ]
    }
   ],
   "source": [
    "# ham 和 spam 示例\n",
    "\n",
    "ham_emails = [load_email(is_spam=False, filename=name) for name in ham_filenames]\n",
    "spam_emails = [load_email(is_spam = True, filename= name) for name in spam_filenames]\n",
    "\n",
    "print(ham_emails[3].get_content().strip())  # get_content 获取邮件  strip去除两头空格"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 电子邮件包含很多部分（图像等），查看邮件类型结构\n",
    "def get_email_structure(email):\n",
    "    if isinstance(email, str):  # 如果只有字符串，说明结构简单\n",
    "        return email\n",
    "    payload = email.get_payload()  # 获取主内容\n",
    "    \n",
    "    if isinstance(email, list) : # 如果是列表，说明组成部分较多\n",
    "        return \"multipart({})\".format(\", \".join([     # multipart 多个部分\n",
    "            get_email_structure(sub_email)\n",
    "            for sub_email in payload               # 列表生成式 + 递归\n",
    "        ]))\n",
    "    else:\n",
    "        return email.get_content_type()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 5, 5: 3, 4: 2, 3: 2, 6: 1, 2: 1})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# contour 方法举例\n",
    "from collections  import Counter\n",
    "a = [1,5,1,5,4,3,1,6,1,4,5,3,1,2]\n",
    "b  = Counter(a) # 求出了每个数字出现的次数， 并以字典类型记录\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "def structures_counter(emails):\n",
    "    structures = Counter()\n",
    "    for email in emails:\n",
    "        structure = get_email_structure(email)\n",
    "        structures[structure] +=1\n",
    "    return structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'text/plain': 2408,\n",
       "         'multipart/signed': 68,\n",
       "         'multipart/alternative': 9,\n",
       "         'multipart/mixed': 10,\n",
       "         'multipart/related': 3,\n",
       "         'multipart/report': 2})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(ham_emails)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('text/plain', 218),\n",
       " ('text/html', 183),\n",
       " ('multipart/alternative', 47),\n",
       " ('multipart/mixed', 43),\n",
       " ('multipart/related', 9)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structures_counter(spam_emails).most_common()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正常邮件与垃圾邮件区别\n",
    "    正常邮件以纯文本为主，垃圾邮件多包涵HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'RE: Important Information Concerning Your Bank Account '"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看邮件头\n",
    "# for header, value in spam_emails[8].items():\n",
    "#     print(header,\":\", value)\n",
    "    \n",
    "spam_emails[8][\"Subject\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拆分训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\wjp\\anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "C:\\Users\\wjp\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = np.array(ham_emails + spam_emails)\n",
    "y = np.array([0] * len(ham_emails) + [1] * len(spam_emails))\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    用[Beautifulsoup]库 将html 转化为 纯文本， 删除`<head>`部分，\n",
    "    然后将所有`<a>`标记转换为单词hyperlink，然后去掉所有html标记，只留下纯文本。\n",
    "    为了可读性，它还用一个换行符替换多个换行符，最后它取消了HTML实体（例如`&gt；`或`&nbsp；`）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from html import unescape\n",
    "\n",
    "def html_to_plain_text(html):\n",
    "    text = re.sub('<head.*?>.*?</head>', '', html, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<a\\s.*?>', ' HYPERLINK ', text, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<.*?>', '', text, flags=re.M | re.S)\n",
    "    text = re.sub(r'(\\s*\\n)+', '\\n', text, flags=re.M | re.S)\n",
    "    return unescape(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html>\n",
      "\n",
      "<head>\n",
      "<meta http-equiv=\"Content-Language\" content=\"en-us\">\n",
      "<meta name=\"GENERATOR\" content=\"Microsoft FrontPage 4.0\">\n",
      "<meta name=\"ProgId\" content=\"FrontPage.Editor.Document\">\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=windows-1252\">\n",
      "<title>Norton AD</title>\n",
      "</head>\n",
      "\n",
      "<body>\n",
      "\n",
      "<table border=\"0\" width=\"489\" height=\"54\" bgcolor=\"#FFFFFF\">\n",
      "  <tr>\n",
      "    <td width=\"100%\" height=\"48\" bgcolor=\"#FFFFFF\" align=\"center\"><font face=\"Impact\" color=\"#CC0000\" size=\"3\">Take\n",
      "      Control of Your Computer With This Top-of-the-Line Software!</font></td>\n",
      "  </tr>\n",
      "</table>\n",
      "\n",
      "<table border=\"5\" cellspacing=\"3\" width=\"489\" cellpadding=\"3\" bgcolor=\"#990033\"  bordercolor=\"#FFFFFF\" style=\"border-collapse: collapse\" bordercolordark=\"#000000\" height=\"567\" bordercolorlight=\"#FFFF99\">\n",
      "  <tr>\n",
      "    <td width=\"467\" height=\"555\" bordercolorlight=\"#FFFFFF\" bordercolordark=\"#FFFF99\">\n",
      "      <table border=\"6\" width=\"100%\" height=\"86\" bgcolor=\"#FFFF99\" bordercolor=\"#FFFFFF\">\n",
      "        <tr>\n",
      "          <td widt ...\n"
     ]
    }
   ],
   "source": [
    "html_spam_emails = [email for email in X_train[y_train==1]\n",
    "                    if get_email_structure(email) == \"text/html\"]  # 取出html类邮件\n",
    "sample_html_spam = html_spam_emails[65]  \n",
    "print(sample_html_spam.get_content().strip()[:1000], \"...\")   \n",
    "# 取其中一个文件  去除两端空格，前一千数据，以...结尾"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    Take\n",
      "      Control of Your Computer With This Top-of-the-Line Software!\n",
      "          Norton\n",
      "            SystemWorks 2002  Software Suite\n",
      "            -Professional Edition-\n",
      "          Includes\n",
      "            Six - Yes 6!\n",
      "            - Feature-Packed UtilitiesALL for 1\n",
      "            Special LOW\n",
      "            Price of Only\n",
      "            $29.99!\n",
      "          This\n",
      "            Software Will: -\n",
      "            Protect your computer from unwanted and  hazardous  viruses -\n",
      "            Help secure your private & valuable information - Allow\n",
      "            you to transfer files and send e-mails  safely -\n",
      "            Backup your ALL your data quick and easily - Improve your\n",
      "            PC's performance w/superior\n",
      "            integral diagnostics! - You'll NEVER have to take your\n",
      "            PC to the repair shop AGAIN!\n",
      "       \n",
      "            6\n",
      "            Feature-Packed Utilities\n",
      "            1\n",
      "            Great Price\n",
      "            A $300+\n",
      "            Combined Retail Value\n",
      "            YOURS  for\n",
      "            Only $29.99!\n",
      " ...\n"
     ]
    }
   ],
   "source": [
    "print(html_to_plain_text(sample_html_spam.get_content())[:1000], \"...\")  # HTML 转化为纯文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 函数，它以  电子邮件为  输入，并以  纯文本形式  返回其内容，无论其格式是什么\n",
    "def email_to_text(email):\n",
    "    html = None\n",
    "    for part in email.walk():\n",
    "        ctype = part.get_content_type()\n",
    "        if not ctype in (\"text/plain\", \"text/html\"):\n",
    "            continue\n",
    "        try:\n",
    "            content = part.get_content()\n",
    "        except: # 解决编码问题\n",
    "            content = str(part.get_payload())\n",
    "        if ctype == \"text/plain\":\n",
    "            return content\n",
    "        else:\n",
    "            html = content\n",
    "    if html:\n",
    "        return html_to_plain_text(html)    #  html 转文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    Take\n",
      "      Control of Your Computer With This Top-of-the-Line Software!\n",
      "          Norton\n",
      "            SystemWorks 2002  Software Suite\n",
      "            -Professional Edition-\n",
      "          Includes\n",
      "            Six - Yes 6!\n",
      "            - Feature-Packed UtilitiesALL for 1\n",
      "            Special LOW\n",
      "            Price of Only\n",
      "            $29.99!\n",
      "          This\n",
      "            Software Will: -\n",
      "            Protect your computer from unwanted and  hazardous  viruses -\n",
      "            Help secure your private & valuabl ...\n"
     ]
    }
   ],
   "source": [
    "print(email_to_text(sample_html_spam)[:500], \"...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    邮件内许多数据对分类无用，比如url、数字字等等\n",
    "    对具体词而言，词的各种变体（-de, ing 等等）转化为一个  （使用自然语言处理工具）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computations => comput\n",
      "Computation => comput\n",
      "Computing => comput\n",
      "Computed => comput\n",
      "Compute => comput\n",
      "Compulsive => compuls\n"
     ]
    }
   ],
   "source": [
    "# “url”替代具体url\n",
    "\n",
    "import nltk\n",
    "from urlextract import URLExtract\n",
    "\n",
    "try:\n",
    "    import nltk\n",
    "\n",
    "    stemmer = nltk.PorterStemmer()\n",
    "    for word in (\"Computations\", \"Computation\", \"Computing\", \"Computed\", \"Compute\", \"Compulsive\"):\n",
    "        print(word, \"=>\", stemmer.stem(word))\n",
    "except ImportError:\n",
    "    print(\"Error: stemming requires the NLTK module.\")\n",
    "    stemmer = None\n",
    "# 将单词各变体  提取词干  前五个都为compute，\n",
    "# 最后一个为compuls类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    将所有处理流程封装为一个转换器，将电子邮件转换为文字计数器。\n",
    "    使用python的'split（）'方法将句子拆分为单词，该方法使用空格作为单词边界。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class EmailToWordCounterTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, strip_headers=True, lower_case=True, remove_punctuation=True,\n",
    "                 replace_urls=True, replace_numbers=True,  stemming=True):\n",
    "        self.strip_headers = strip_headers  # 是否去除头部\n",
    "        self.lower_case = lower_case        # 小写\n",
    "        self.remove_punctuation = remove_punctuation   # 标点符号\n",
    "        self.replace_urls = replace_urls             # url\n",
    "        self.replace_numbers = replace_numbers        # 数字\n",
    "        self.stemming = stemming                    # 提取词干？\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X_transformed = []\n",
    "        for email in X:\n",
    "            text = email_to_text(email) or \"\"         # 将邮件转化为 纯文本\n",
    "            if self.lower_case:\n",
    "                text = text.lower()\n",
    "            if self.replace_urls:\n",
    "                extractor = URLExtract()\n",
    "                urls = list(set(extractor.find_urls(text)))\n",
    "                urls.sort(key=lambda url: len(url), reverse=True)\n",
    "                for url in urls:      # 替换url 为 ‘URL’\n",
    "                    text = text.replace(url, \" URL \")\n",
    "            if self.replace_numbers:    # 替换数字\n",
    "                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?', 'NUMBER', text)\n",
    "            if self.remove_punctuation:   # 删除标点符号\n",
    "                text = re.sub(r'\\W+', ' ', text, flags=re.M)\n",
    "            word_counts = Counter(text.split())    # 使用空格分词  并统计\n",
    "            if self.stemming and stemmer is not None:   # 提取词干 并统计\n",
    "                stemmed_word_counts = Counter()\n",
    "                for word, count in word_counts.items():\n",
    "                    stemmed_word = stemmer.stem(word)\n",
    "                    stemmed_word_counts[stemmed_word] += count\n",
    "                word_counts = stemmed_word_counts\n",
    "            X_transformed.append(word_counts)\n",
    "        return np.array(X_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([Counter({'chuck': 1, 'murcko': 1, 'wrote': 1, 'stuff': 1, 'yawn': 1, 'r': 1}),\n",
       "       Counter({'the': 11, 'of': 9, 'and': 8, 'all': 3, 'christian': 3, 'to': 3, 'by': 3, 'jefferson': 2, 'i': 2, 'have': 2, 'superstit': 2, 'one': 2, 'on': 2, 'been': 2, 'ha': 2, 'half': 2, 'rogueri': 2, 'teach': 2, 'jesu': 2, 'some': 1, 'interest': 1, 'quot': 1, 'url': 1, 'thoma': 1, 'examin': 1, 'known': 1, 'word': 1, 'do': 1, 'not': 1, 'find': 1, 'in': 1, 'our': 1, 'particular': 1, 'redeem': 1, 'featur': 1, 'they': 1, 'are': 1, 'alik': 1, 'found': 1, 'fabl': 1, 'mytholog': 1, 'million': 1, 'innoc': 1, 'men': 1, 'women': 1, 'children': 1, 'sinc': 1, 'introduct': 1, 'burnt': 1, 'tortur': 1, 'fine': 1, 'imprison': 1, 'what': 1, 'effect': 1, 'thi': 1, 'coercion': 1, 'make': 1, 'world': 1, 'fool': 1, 'other': 1, 'hypocrit': 1, 'support': 1, 'error': 1, 'over': 1, 'earth': 1, 'six': 1, 'histor': 1, 'american': 1, 'john': 1, 'e': 1, 'remsburg': 1, 'letter': 1, 'william': 1, 'short': 1, 'again': 1, 'becom': 1, 'most': 1, 'pervert': 1, 'system': 1, 'that': 1, 'ever': 1, 'shone': 1, 'man': 1, 'absurd': 1, 'untruth': 1, 'were': 1, 'perpetr': 1, 'upon': 1, 'a': 1, 'larg': 1, 'band': 1, 'dupe': 1, 'import': 1, 'led': 1, 'paul': 1, 'first': 1, 'great': 1, 'corrupt': 1}),\n",
       "       Counter({'url': 4, 's': 3, 'group': 3, 'to': 3, 'in': 2, 'forteana': 2, 'martin': 2, 'an': 2, 'and': 2, 'we': 2, 'is': 2, 'yahoo': 2, 'unsubscrib': 2, 'y': 1, 'adamson': 1, 'wrote': 1, 'for': 1, 'altern': 1, 'rather': 1, 'more': 1, 'factual': 1, 'base': 1, 'rundown': 1, 'on': 1, 'hamza': 1, 'career': 1, 'includ': 1, 'hi': 1, 'belief': 1, 'that': 1, 'all': 1, 'non': 1, 'muslim': 1, 'yemen': 1, 'should': 1, 'be': 1, 'murder': 1, 'outright': 1, 'know': 1, 'how': 1, 'unbias': 1, 'memri': 1, 'don': 1, 't': 1, 'html': 1, 'rob': 1, 'sponsor': 1, 'number': 1, 'dvd': 1, 'free': 1, 'p': 1, 'join': 1, 'now': 1, 'from': 1, 'thi': 1, 'send': 1, 'email': 1, 'egroup': 1, 'com': 1, 'your': 1, 'use': 1, 'of': 1, 'subject': 1}),\n",
       "       Counter({'to': 6, 'anthoni': 2, 'i': 2, 'url': 2, 'a': 2, 'or': 2, 'it': 2, 'skip': 1, 'montanaro': 1, 'baxter': 1, 'accordingli': 1, 'wrote': 1, 'which': 1, 'is': 1, 'mostli': 1, 'ripoff': 1, 'of': 1, 'someth': 1, 'someon': 1, 'els': 1, 'post': 1, 'python': 1, 'dev': 1, 'within': 1, 'the': 1, 'last': 1, 'week': 1, 'so': 1, 'strip': 1, 'out': 1, 'sa': 1, 'gener': 1, 'header': 1, 'unless': 1, 've': 1, 'grown': 1, 'senil': 1, 'tonight': 1, 'you': 1, 'got': 1, 'from': 1, 'begin': 1, 'with': 1, 'pleas': 1, 'check': 1, 'in': 1, 'project': 1, 'and': 1, 'add': 1, 'short': 1, 'blurb': 1, 'readm': 1, 'txt': 1})],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试转换器\n",
    "X_few = X_train[:4]\n",
    "X_few_wordcounts = EmailToWordCounterTransformer().fit_transform(X_few)\n",
    "X_few_wordcounts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    有了单词计数，我们需要把它们转换成向量。为此，我们将构建另一个转换器，\n",
    "    其“fit（）”方法将构建词汇表（最常用单词的有序列表），\n",
    "    其“transform（）”方法将使用词汇表将单词计数转换为向量--稀疏矩阵\n",
    "    稀疏矩阵运算量大，存储占空间，需进行压缩"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, vocabulary_size = 1000):\n",
    "        self.vocabulary_size = vocabulary_size    # 词汇量 默认1000\n",
    "    def fit(self, X, y = None):\n",
    "        total_count = Counter()                   # 字典，单词：出现次数  （total_count统计全部邮件）\n",
    "        for word_count in X:                      # word_count  为每个单词出现的次数 （为一个字典，例'in': 1 ）\n",
    "            for word, count in word_count.items():  \n",
    "                total_count[word] += min(count, 10)   # 设置数据上限为10\n",
    "        most_common = total_count.most_common()[:self.vocabulary_size]\n",
    "        self.most_common_ = most_common\n",
    "        self.vocabulary_ = {word: index + 1 for index, (word, count) in enumerate(most_common)}\n",
    "        return self\n",
    "    def transform(self, X, y = None):       # 转化为稀疏矩阵\n",
    "        rows = []\n",
    "        cols = []\n",
    "        data = []\n",
    "        for row, word_count in enumerate(X):     # row 为邮件索引， word_count 为邮件名\n",
    "            for word, count in word_count.items():     # word ， \n",
    "                rows.append(row) # 训练集 实例个数\n",
    "                cols.append(self.vocabulary_.get(word, 0)) # 取得单词在词汇表中的索引位置，0代表未出现在词汇表中\n",
    "                data.append(count)\n",
    "        return csr_matrix((data, (rows, cols)), shape=(len(X), self.vocabulary_size + 1))\n",
    "    # 输出稀疏矩阵 +1因为第一列要显示未出现在词汇表中的单词统计数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 0)\t1\n",
      "  (0, 1)\t0\n",
      "  (0, 2)\t2\n",
      "  (1, 0)\t0\n",
      "  (1, 1)\t8\n",
      "  (1, 2)\t1\n",
      "  (2, 0)\t6\n",
      "  (2, 1)\t1\n",
      "  (2, 2)\t0\n",
      "[[1 0 2]\n",
      " [0 8 1]\n",
      " [6 1 0]]\n"
     ]
    }
   ],
   "source": [
    "from scipy.sparse import *\n",
    " \n",
    "row =  [0,0,0,1,1,1,2,2,2] #行指标\n",
    "col =  [0,1,2,0,1,2,0,1,2] #列指标\n",
    "data = [1,0,2,0,8,1,6,1,0] #在行指标列指标下的数字\n",
    "team = csr_matrix((data,(row,col)),shape=(3,3))  # 压缩稀疏矩阵\n",
    "print(team)\n",
    "print(team.todense())\n",
    "# team.toarray()   # 返回稀疏矩阵\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, 16)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer = WordCounterToVectorTransformer(vocabulary_size=15)\n",
    "X_few_vectors = vocab_transformer.fit_transform(X_few_wordcounts)\n",
    "X_few_vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5,  0,  0,  0,  0,  0,  0,  0,  0,  1,  0,  0,  0,  0,  0,  0],\n",
       "       [96,  3, 11,  9,  8,  1,  2,  3,  1,  0,  3,  2,  3,  1,  0,  0],\n",
       "       [61,  3,  0,  1,  2,  4,  0,  1,  2,  1,  0,  1,  0,  0,  3,  2],\n",
       "       [46,  6,  1,  1,  1,  2,  2,  0,  1,  1,  0,  0,  0,  2,  0,  1]],\n",
       "      dtype=int32)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_few_vectors.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    第二个邮件，96个词不在词汇表（词汇表包含15个单词）中，3个‘to’，11个‘the’"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'to': 1,\n",
       " 'the': 2,\n",
       " 'of': 3,\n",
       " 'and': 4,\n",
       " 'url': 5,\n",
       " 'i': 6,\n",
       " 'all': 7,\n",
       " 'in': 8,\n",
       " 'wrote': 9,\n",
       " 'christian': 10,\n",
       " 'on': 11,\n",
       " 'by': 12,\n",
       " 'a': 13,\n",
       " 's': 14,\n",
       " 'is': 15}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transformer.vocabulary_   # 词汇表 排名"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    训练第一个垃圾邮件分类器,让我们转换整个数据集："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2400x1001 sparse matrix of type '<class 'numpy.intc'>'\n",
       "\twith 188522 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "preprocess_pipeline = Pipeline([\n",
    "    (\"email_to_wordcount\", EmailToWordCounterTransformer()),\n",
    "    (\"wordcount_to_verctor\", WordCounterToVectorTransformer()),\n",
    "])\n",
    "\n",
    "X_train_transformer = preprocess_pipeline.fit_transform(X_train)\n",
    "X_train_transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END ................................ score: (test=0.983) total time=   0.0s\n",
      "[CV] END ................................ score: (test=0.988) total time=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.1s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9854166666666666"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression   # 逻辑回归分类器\n",
    "from sklearn.model_selection import cross_val_score   # 交叉验证\n",
    "\n",
    "log_clf =  LogisticRegression(solver = \"liblinear\", random_state = 42) \n",
    "score = cross_val_score(log_clf, X_train_transformer, y_train, cv = 2, verbose = 3)\n",
    "score.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "尝试多个模型，使用交叉验证对其进行微调。在测试集上获取精度和召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "精度: 96.88%\n",
      "召回: 97.89%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "X_test_transformed = preprocess_pipeline.transform(X_test)\n",
    "\n",
    "log_clf = LogisticRegression(solver = \"liblinear\", random_state=42)\n",
    "log_clf.fit(X_train_transformer, y_train)\n",
    "\n",
    "y_pred = log_clf.predict(X_test_transformed)\n",
    "\n",
    "print(\"精度: {:.2f}%\".format(100 * precision_score(y_test, y_pred)))\n",
    "print(\"召回: {:.2f}%\".format(100 * recall_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\wjp\\anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(random_state=42)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor   # 使用决策树\n",
    "\n",
    "tree_reg = DecisionTreeRegressor(random_state=42)\n",
    "tree_reg.fit(X_train_transformer, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "精度: 91.49%\n",
      "召回: 90.53%\n"
     ]
    }
   ],
   "source": [
    "y_pred = tree_reg.predict(X_test_transformed)\n",
    "print(\"精度: {:.2f}%\".format(100 * precision_score(y_test, y_pred)))\n",
    "print(\"召回: {:.2f}%\".format(100 * recall_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
